"clickhouse": {
	annotations: {}
	labels: {}
	attributes: {
		"dynamicParameterMeta": [
      {
				"name":        "dependencies.zookeeperHost"
				"type":        "ContextSetting"
				"refType":     "zookeeper"
				"refKey":      "hostname"
				"description": "zookeeper host"
				"required":    true
			}
		]
		apiResource: {
			definition: {
				apiVersion: "bdc.kdp.io/v1alpha1"
				kind:       "Application"
				type:       "clickhouse"
			}
		}
	}
	description: "clickhouse xdefinition"
	type:        "xdefinition"
}

template: {
	output: {
		apiVersion: "core.oam.dev/v1beta1"
		kind:       "Application"
		metadata: {
			name:      context.name
			namespace: context.namespace
			labels: {
				"app":                context.name
				"app.core.bdos/type": "system"
			}
		}
		spec: {
			components: [
				{
					name: context.name
					type: "helm"
					properties: {
						chart:           "clickhouse"
						version:         parameter.chartVersion
						url:             context.helm_repo_url
						repoType:        "oci"
						releaseName:     context.name
						targetNamespace: context.namespace
						values: {
              image: {
                registry: context.docker_registry
                tag: parameter.imageTag
              }
              shards: parameter.shards
              replicaCount: parameter.replicaCount
              resources: parameter.resources
              auth: {
                username: parameter.auth.username
                password: parameter.auth.password
              }
              podAntiAffinityPreset: "soft"
              persistence: {
                enabled: true
                storageClass: context["storage_config.storage_class_mapping.local_disk"]
                size: parameter.storageSize
              }
              metrics: enabled: true
              zookeeper: enabled: false
              externalZookeeper: {
                servers: [
                  parameter.dependencies.zookeeperHost
                ]
                port: 2181
              }
            }
					}
					traits: [
						{
							type: "bdos-monitor"
							properties: {
								monitortype: "pod"
								endpoints: [
									{
										port:     8001
										portName: "http-metrics"
									},
								]
								matchLabels: {
									"app.kubernetes.io/instance": context.name
								}
							}
						}, 
						{
							type: "bdos-grafana-dashboard"
							properties: {
								labels: {
									"app":               "grafana"
									"grafana_dashboard": "1"
								}
								dashboard_data: {
									"clickhouse-dashboard.json": #"{"annotations":{"list":[{"builtIn":1,"datasource":{"type":"datasource","uid":"grafana"},"enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations & Alerts","type":"dashboard"},{"datasource":{"type":"prometheus","uid":"prometheus"},"enable":true,"expr":"resets(ClickHouseAsyncMetrics_Uptime{pod=~\"$pod\",namespace=\"$namespace\"}[$__rate_interval])","hide":false,"iconColor":"rgba(255, 96, 96, 1)","name":"Restarts","showIn":0,"step":"60s","tagKeys":"instance","titleFormat":"Restart","useValueForTime":false}]},"description":"ClickHouse internal exporter metrics","editable":true,"fiscalYearStartMonth":0,"gnetId":14432,"graphTooltip":1,"id":72,"links":[{"asDropdown":false,"icon":"external link","includeVars":false,"keepTime":false,"tags":[],"targetBlank":true,"title":"ClickHouse monitoring docs","tooltip":"","type":"link","url":"https://clickhouse.tech/docs/en/operations/monitoring/"}],"liveNow":false,"panels":[{"collapsed":false,"datasource":{"type":"prometheus","uid":"prometheus"},"gridPos":{"h":1,"w":24,"x":0,"y":0},"id":136,"panels":[],"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"refId":"A"}],"title":"Baisc","type":"row"},{"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"decimals":1,"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"semi-dark-red","value":null},{"color":"semi-dark-orange","value":300},{"color":"light-green","value":3600}]},"unit":"s","unitScale":true},"overrides":[]},"gridPos":{"h":4,"w":8,"x":0,"y":1},"id":132,"options":{"colorMode":"value","graphMode":"none","justifyMode":"auto","orientation":"horizontal","reduceOptions":{"calcs":["mean"],"fields":"","values":false},"showPercentChange":false,"textMode":"auto","wideLayout":true},"pluginVersion":"10.3.3","targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"editorMode":"code","expr":"ClickHouseAsyncMetrics_Uptime{pod=~\"$pod\",namespace=\"$namespace\"}","interval":"","legendFormat":"","range":true,"refId":"A"}],"title":"UpTime","type":"stat"},{"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"semi-dark-red","value":null},{"color":"semi-dark-orange","value":300},{"color":"light-green","value":3600}]},"unit":"none","unitScale":true},"overrides":[]},"gridPos":{"h":4,"w":8,"x":8,"y":1},"id":133,"options":{"colorMode":"value","graphMode":"none","justifyMode":"auto","orientation":"horizontal","reduceOptions":{"calcs":["mean"],"fields":"","values":false},"showPercentChange":false,"textMode":"auto","wideLayout":true},"pluginVersion":"10.3.3","targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"editorMode":"code","expr":"ClickHouseMetrics_VersionInteger{pod=~\"$pod\",namespace=\"$namespace\"}","interval":"","legendFormat":"","range":true,"refId":"A"}],"title":"Version","type":"stat"},{"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"semi-dark-red","value":null},{"color":"semi-dark-orange","value":300},{"color":"light-green","value":3600}]},"unit":"none","unitScale":true},"overrides":[]},"gridPos":{"h":4,"w":8,"x":16,"y":1},"id":134,"options":{"colorMode":"value","graphMode":"none","justifyMode":"auto","orientation":"auto","reduceOptions":{"calcs":["mean"],"fields":"","values":false},"showPercentChange":false,"textMode":"auto","wideLayout":true},"pluginVersion":"10.3.3","targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"editorMode":"code","expr":"count(clickhouse_table_parts_count{database!~\"system\",pod=~\"$pod\",namespace=\"$namespace\"})","interval":"","legendFormat":"","range":true,"refId":"A"}],"title":"Tables","type":"stat"},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"Total amount of memory (bytes) allocated by the server.","fieldConfig":{"defaults":{"unitScale":true},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":7,"w":12,"x":0,"y":5},"hiddenSeries":false,"id":29,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"10.3.3","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"editorMode":"code","exemplar":false,"expr":"ClickHouseMetrics_MemoryTracking{pod=~\"$pod\",namespace=\"$namespace\"}","interval":"","legendFormat":"{{pod}}","range":true,"refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Memory","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"bytes","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"Number of connections to TCP server (clients with native interface), also included server-server distributed query connections. \nNumber of connections to HTTP server","fieldConfig":{"defaults":{"unitScale":true},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":12,"y":5},"hiddenSeries":false,"id":73,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"10.3.3","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"ClickHouseMetrics_TCPConnection{pod=~\"$pod\",namespace=\"$namespace\"}","interval":"","legendFormat":"tcp","refId":"A"},{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"ClickHouseMetrics_HTTPConnection{pod=~\"$pod\",namespace=\"$namespace\"}","hide":false,"interval":"","legendFormat":"http","refId":"B"},{"datasource":{"type":"prometheus","uid":"prometheus"},"expr":"ClickHouseMetrics_InterserverConnection{pod=~\"$pod\",namespace=\"$namespace\"}","interval":"","legendFormat":"interserver","refId":"C"},{"datasource":{"type":"prometheus","uid":"prometheus"},"expr":"ClickHouseMetrics_MySQLConnection{pod=~\"$pod\",namespace=\"$namespace\"}","interval":"","legendFormat":"mysql","refId":"D"},{"datasource":{"type":"prometheus","uid":"prometheus"},"expr":"ClickHouseMetrics_PostgreSQLConnection{pod=~\"$pod\",namespace=\"$namespace\"}","interval":"","legendFormat":"postgre sql","refId":"E"}],"thresholds":[],"timeRegions":[],"title":"Connections","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"short","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"collapsed":false,"datasource":{"type":"prometheus","uid":"prometheus"},"gridPos":{"h":1,"w":24,"x":0,"y":13},"id":12,"panels":[],"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"refId":"A"}],"title":"Queries","type":"row"},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"Number of queries to be interpreted and potentially executed. Does not include queries that failed to parse or were rejected due to AST size limits, quota limits or limits on the number of simultaneously running queries. May include internal queries initiated by ClickHouse itself. Does not count subqueries.","fieldConfig":{"defaults":{"unitScale":true},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":0,"y":14},"hiddenSeries":false,"id":2,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"10.3.3","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"max_over_time( (irate(ClickHouseProfileEvents_Query{pod=~\"$pod\",namespace=\"$namespace\"}[2m]))[$__rate_interval:15s] )","interval":"","legendFormat":"queries","refId":"A"},{"datasource":{"type":"prometheus","uid":"prometheus"},"expr":"max_over_time( (irate(ClickHouseProfileEvents_FailedQuery{pod=~\"$pod\",namespace=\"$namespace\"}[2m]))[$__rate_interval:15s] )","interval":"","legendFormat":"failed queries","refId":"B"}],"thresholds":[],"timeRegions":[],"title":"Queries","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"reqps","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"Avg queries latencies","fieldConfig":{"defaults":{"unitScale":true},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":12,"y":14},"hiddenSeries":false,"id":8,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null as zero","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"10.3.3","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"increase(ClickHouseProfileEvents_QueryTimeMicroseconds{pod=~\"$pod\",namespace=\"$namespace\"}[$__rate_interval]) / increase(ClickHouseProfileEvents_Query{pod=~\"$pod\",namespace=\"$namespace\"}[$__rate_interval])","interval":"","legendFormat":"all queries","refId":"A"},{"datasource":{"type":"prometheus","uid":"prometheus"},"expr":"increase(ClickHouseProfileEvents_InsertQueryTimeMicroseconds{pod=~\"$pod\",namespace=\"$namespace\"}[$__rate_interval]) / increase(ClickHouseProfileEvents_InsertQuery{pod=~\"$pod\",namespace=\"$namespace\"}[$__rate_interval])","interval":"","legendFormat":"insert queries","refId":"B"},{"datasource":{"type":"prometheus","uid":"prometheus"},"expr":"increase(ClickHouseProfileEvents_SelectQueryTimeMicroseconds{pod=~\"$pod\",namespace=\"$namespace\"}[$__rate_interval]) / increase(ClickHouseProfileEvents_SelectQuery{pod=~\"$pod\",namespace=\"$namespace\"}[$__rate_interval])","interval":"","legendFormat":"select queries","refId":"C"}],"thresholds":[],"timeRegions":[],"title":"Query latencies (avg)","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"µs","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false,"alignLevel":1}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"Same as Queries, but only for SELECT queries.","fieldConfig":{"defaults":{"unitScale":true},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":0,"y":22},"hiddenSeries":false,"id":3,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"10.3.3","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"max_over_time( (irate(ClickHouseProfileEvents_SelectQuery{pod=~\"$pod\",namespace=\"$namespace\"}[2m]))[$__rate_interval:15s] )","interval":"","legendFormat":"select queries","refId":"A"},{"datasource":{"type":"prometheus","uid":"prometheus"},"expr":"max_over_time( (irate(ClickHouseProfileEvents_FailedSelectQuery{pod=~\"$pod\",namespace=\"$namespace\"}[2m]))[$__rate_interval:15s] )","interval":"","legendFormat":"failed select queries","refId":"B"}],"thresholds":[],"timeRegions":[],"title":"SELECT queries","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"reqps","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"Same as Queries, but only for INSERT queries.","fieldConfig":{"defaults":{"unitScale":true},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":12,"y":22},"hiddenSeries":false,"id":4,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"10.3.3","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"max_over_time( (irate(ClickHouseProfileEvents_InsertQuery{pod=~\"$pod\",namespace=\"$namespace\"}[2m]))[$__rate_interval:15s] )","interval":"","legendFormat":"insert queries","refId":"A"},{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"max_over_time( (irate(ClickHouseProfileEvents_FailedInsertQuery{pod=~\"$pod\",namespace=\"$namespace\"}[2m]))[$__rate_interval:15s] )","hide":false,"interval":"","legendFormat":"failed insert queries","refId":"B"}],"thresholds":[],"timeRegions":[],"title":"INSERT queries","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"reqps","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"Number of queries that are stopped and waiting due to 'priority' setting.","fieldConfig":{"defaults":{"unitScale":true},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":0,"y":30},"hiddenSeries":false,"id":86,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"10.3.3","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"max_over_time( (irate(ClickHouseMetrics_QueryPreempted{pod=~\"$pod\",namespace=\"$namespace\"}[2m]))[$__rate_interval:15s] )","interval":"","legendFormat":"{{pod}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Query preemted","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"reqps","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"Number of times when memory limit exceeded for query.","fieldConfig":{"defaults":{"unitScale":true},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":12,"y":30},"hiddenSeries":false,"id":85,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"10.3.3","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"max_over_time( (irate(ClickHouseProfileEvents_QueryMemoryLimitExceeded{pod=~\"$pod\",namespace=\"$namespace\"}[2m]))[$__rate_interval:15s] )","interval":"","legendFormat":"{{pod}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Queries memory limit","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"reqps","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"collapsed":true,"datasource":{"type":"prometheus","uid":"prometheus"},"gridPos":{"h":1,"w":24,"x":0,"y":38},"id":64,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"Number of rows INSERTed to all tables.","fieldConfig":{"defaults":{"unitScale":true},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":0,"y":39},"hiddenSeries":false,"id":62,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"10.3.3","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"max_over_time( (irate(ClickHouseProfileEvents_InsertedRows{pod=~\"$pod\",namespace=\"$namespace\"}[2m]))[$__rate_interval:15s] )","interval":"","legendFormat":"{{pod}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Inserted rows","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"cps","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"Number of bytes (uncompressed; for columns as they stored in memory) INSERTed to all tables.","fieldConfig":{"defaults":{"unitScale":true},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":12,"y":39},"hiddenSeries":false,"id":65,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"10.3.3","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"max_over_time( (irate(ClickHouseProfileEvents_InsertedBytes{pod=~\"$pod\",namespace=\"$namespace\"}[2m]))[$__rate_interval:15s] )","interval":"","legendFormat":"{{pod}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Inserted bytes","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"binBps","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"Number of times the INSERT of a block to a MergeTree table was throttled due to high number of active data parts for partition.","fieldConfig":{"defaults":{"unitScale":true},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":0,"y":47},"hiddenSeries":false,"id":66,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"10.3.3","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"max_over_time( (irate(ClickHouseProfileEvents_DelayedInserts{pod=~\"$pod\",namespace=\"$namespace\"}[2m]))[$__rate_interval:15s] )","interval":"","legendFormat":"{{pod}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Delayed inserts","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"cps","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"Number of times the INSERT of a block to a MergeTree table was rejected with 'Too many parts' exception due to high number of active data parts for partition.","fieldConfig":{"defaults":{"unitScale":true},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":12,"y":47},"hiddenSeries":false,"id":67,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"10.3.3","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"max_over_time( (irate(ClickHouseProfileEvents_RejectedInserts{pod=~\"$pod\",namespace=\"$namespace\"}[2m]))[$__rate_interval:15s] )","interval":"","legendFormat":"{{pod}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Rejected inserts","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"cps","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"Total number of milliseconds spent while the INSERT of a block to a MergeTree table was throttled due to high number of active data parts for partition.","fieldConfig":{"defaults":{"unitScale":true},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":0,"y":55},"hiddenSeries":false,"id":69,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null as zero","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"10.3.3","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"increase(ClickHouseProfileEvents_DelayedInsertsMilliseconds{pod=~\"$pod\",namespace=\"$namespace\"}[$__rate_interval]) / increase(ClickHouseProfileEvents_DelayedInserts{pod=~\"$pod\",namespace=\"$namespace\"}[$__rate_interval])","interval":"","legendFormat":"{{pod}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Delayed inserts blocks (avg)","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"µs","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false,"alignLevel":1}}],"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"refId":"A"}],"title":"Insert","type":"row"},{"collapsed":true,"datasource":{"type":"prometheus","uid":"prometheus"},"gridPos":{"h":1,"w":24,"x":0,"y":39},"id":125,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"Number of data parts selected to read from a MergeTree table.","fieldConfig":{"defaults":{"unitScale":true},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":0,"y":4},"hiddenSeries":false,"id":126,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"10.3.3","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"max_over_time( (irate(ClickHouseProfileEvents_SelectedParts{pod=~\"$pod\",namespace=\"$namespace\"}[2m]))[$__rate_interval:15s] )","interval":"","legendFormat":"{{pod}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Parts","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"cps","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"Number of (non-adjacent) ranges in all data parts selected to read from a MergeTree table.","fieldConfig":{"defaults":{"unitScale":true},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":12,"y":4},"hiddenSeries":false,"id":127,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"10.3.3","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"max_over_time( (irate(ClickHouseProfileEvents_SelectedRanges{pod=~\"$pod\",namespace=\"$namespace\"}[2m]))[$__rate_interval:15s] )","interval":"","legendFormat":"{{pod}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Ranges","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"cps","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"Number of marks (index granules) selected to read from a MergeTree table.","fieldConfig":{"defaults":{"unitScale":true},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":0,"y":12},"hiddenSeries":false,"id":128,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"10.3.3","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"max_over_time( (irate(ClickHouseProfileEvents_SelectedMarks{pod=~\"$pod\",namespace=\"$namespace\"}[2m]))[$__rate_interval:15s] )","interval":"","legendFormat":"{{pod}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Marks","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"cps","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"Number of rows SELECTed from all tables.","fieldConfig":{"defaults":{"unitScale":true},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":12,"y":12},"hiddenSeries":false,"id":129,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"10.3.3","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"max_over_time( (irate(ClickHouseProfileEvents_SelectedRows{pod=~\"$pod\",namespace=\"$namespace\"}[2m]))[$__rate_interval:15s] )","interval":"","legendFormat":"{{pod}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Rows","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"cps","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"Number of bytes (uncompressed; for columns as they stored in memory) SELECTed from all tables.","fieldConfig":{"defaults":{"unitScale":true},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":0,"y":20},"hiddenSeries":false,"id":130,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"10.3.3","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"max_over_time( (irate(ClickHouseProfileEvents_SelectedBytes{pod=~\"$pod\",namespace=\"$namespace\"}[2m]))[$__rate_interval:15s] )","interval":"","legendFormat":"{{pod}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Bytes","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"binBps","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}}],"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"refId":"A"}],"title":"Select","type":"row"},{"collapsed":true,"datasource":{"type":"prometheus","uid":"prometheus"},"gridPos":{"h":1,"w":24,"x":0,"y":40},"id":138,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"Number of times the number of query processing threads was lowered due to slow reads.","fieldConfig":{"defaults":{"custom":{}},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":0,"y":17},"hiddenSeries":false,"id":72,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.3.5","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"max_over_time( (irate(ClickHouseProfileEvents_ReadBackoff{pod=~\"$pod\",namespace=\"$namespace\"}[2m]))[$__rate_interval:15s] )","interval":"","legendFormat":"{{pod}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Read backoff","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"cps","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"Number of reads from a file that were slow. This indicate system overload. Thresholds are controlled by read_backoff_* settings.","fieldConfig":{"defaults":{"custom":{}},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":7,"w":12,"x":12,"y":17},"hiddenSeries":false,"id":71,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.3.5","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"max_over_time( (irate(ClickHouseProfileEvents_SlowRead{pod=~\"$pod\",namespace=\"$namespace\"}[2m]))[$__rate_interval:15s] )","interval":"","legendFormat":"{{pod}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Slow reads","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"cps","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}}],"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"refId":"A"}],"title":"Read","type":"row"},{"collapsed":true,"datasource":{"type":"prometheus","uid":"prometheus"},"gridPos":{"h":1,"w":24,"x":0,"y":41},"id":17,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"Number of times the 'lseek' function was called.","fieldConfig":{"defaults":{"custom":{}},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":0,"y":6},"hiddenSeries":false,"id":20,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.3.5","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"max_over_time( (irate(ClickHouseProfileEvents_Seek{pod=~\"$pod\",namespace=\"$namespace\"}[2m]))[$__rate_interval:15s] )","interval":"","legendFormat":"{{pod}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"lseek","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"ops","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"Number of files opened","fieldConfig":{"defaults":{"custom":{}},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":12,"y":6},"hiddenSeries":false,"id":19,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.3.5","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"max_over_time( (irate(ClickHouseProfileEvents_FileOpen{pod=~\"$pod\",namespace=\"$namespace\"}[2m]))[$__rate_interval:15s] )","interval":"","legendFormat":"{{pod}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Open Files","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"ops","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"Number of reads (read/pread) from a file descriptor. Does not include sockets.","fieldConfig":{"defaults":{"custom":{}},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":0,"y":14},"hiddenSeries":false,"id":22,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.3.5","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"max_over_time( (irate(ClickHouseProfileEvents_ReadBufferFromFileDescriptorRead{pod=~\"$pod\",namespace=\"$namespace\"}[2m]))[$__rate_interval:15s] )","interval":"","legendFormat":"read","refId":"A"},{"datasource":{"type":"prometheus","uid":"prometheus"},"expr":"max_over_time( (irate(ClickHouseProfileEvents_ReadBufferFromFileDescriptorReadFailed{pod=~\"$pod\",namespace=\"$namespace\"}[2m]))[$__rate_interval:15s] )","interval":"","legendFormat":"read failed","refId":"B"}],"thresholds":[],"timeRegions":[],"title":"Read from FD","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"ops","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"Number of writes (write/pwrite) to a file descriptor. Does not include sockets.","fieldConfig":{"defaults":{"custom":{}},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":12,"y":14},"hiddenSeries":false,"id":23,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.3.5","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"max_over_time( (irate(ClickHouseProfileEvents_WriteBufferFromFileDescriptorWrite{pod=~\"$pod\",namespace=\"$namespace\"}[2m]))[$__rate_interval:15s] )","interval":"","legendFormat":"write","refId":"A"},{"datasource":{"type":"prometheus","uid":"prometheus"},"expr":"max_over_time( (irate(ClickHouseProfileEvents_WriteBufferFromFileDescriptorWriteFailed{pod=~\"$pod\",namespace=\"$namespace\"}[2m]))[$__rate_interval:15s] )","interval":"","legendFormat":"write failed","refId":"B"}],"thresholds":[],"timeRegions":[],"title":"Write to FD","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"ops","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"Number of bytes read from file descriptors. If the file is compressed, this will show the compressed data size.","fieldConfig":{"defaults":{"custom":{}},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":0,"y":22},"hiddenSeries":false,"id":21,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.3.5","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"max_over_time( (irate(ClickHouseProfileEvents_ReadBufferFromFileDescriptorReadBytes{pod=~\"$pod\",namespace=\"$namespace\"}[2m]))[$__rate_interval:15s] )","interval":"","legendFormat":"{{pod}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Read bytes from FD","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"binBps","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"Number of bytes written to file descriptors. If the file is compressed, this will show compressed data size.","fieldConfig":{"defaults":{"custom":{}},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":12,"y":22},"hiddenSeries":false,"id":26,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.3.5","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"max_over_time( (irate(ClickHouseProfileEvents_WriteBufferFromFileDescriptorWriteBytes{pod=~\"$pod\",namespace=\"$namespace\"}[2m]))[$__rate_interval:15s] )","interval":"","legendFormat":"{{pod}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Write bytes to FD","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"binBps","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"*``compressed``: Number of bytes (the number of bytes before decompression) read from compressed sources (files, network).\n\n*``uncompressed``: Number of uncompressed bytes (the number of bytes after decompression) read from compressed sources (files, network).","fieldConfig":{"defaults":{"custom":{}},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":0,"y":30},"hiddenSeries":false,"id":51,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"sort":"max","sortDesc":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.3.5","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"max_over_time( (irate(ClickHouseProfileEvents_ReadCompressedBytes{pod=~\"$pod\",namespace=\"$namespace\"}[2m]))[$__rate_interval:15s] )","interval":"","legendFormat":"compressed","refId":"A"},{"datasource":{"type":"prometheus","uid":"prometheus"},"expr":"max_over_time( (irate(ClickHouseProfileEvents_CompressedReadBufferBytes{pod=~\"$pod\",namespace=\"$namespace\"}[2m]))[$__rate_interval:15s] )","interval":"","legendFormat":"uncompressed","refId":"B"}],"thresholds":[],"timeRegions":[],"title":"Read compressed bytes","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"binBps","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"Number of compressed blocks (the blocks of data that are compressed independent of each other) read from compressed sources (files, network).","fieldConfig":{"defaults":{"custom":{}},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":12,"y":30},"hiddenSeries":false,"id":52,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.3.5","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"max_over_time( (irate(ClickHouseProfileEvents_CompressedReadBufferBlocks{pod=~\"$pod\",namespace=\"$namespace\"}[2m]))[$__rate_interval:15s] )","interval":"","legendFormat":"peaks - {{instance}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Compressed blocks","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"cps","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}}],"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"refId":"A"}],"title":"IO","type":"row"},{"collapsed":true,"datasource":{"type":"prometheus","uid":"prometheus"},"gridPos":{"h":1,"w":24,"x":0,"y":42},"id":56,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"Number of Replicated tables that are currently in readonly state due to re-initialization after ZooKeeper session loss or due to startup without ZooKeeper configured.","fieldConfig":{"defaults":{"custom":{}},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":0,"y":7},"hiddenSeries":false,"id":74,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.3.5","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"ClickHouseMetrics_ReadonlyReplica","interval":"","legendFormat":"{{pod}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Readonly replica","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"short","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"Number of times a data part was failed to download from replica of a ReplicatedMergeTree table.","fieldConfig":{"defaults":{"custom":{}},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":12,"y":7},"hiddenSeries":false,"id":58,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.3.5","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"max_over_time( (irate(ClickHouseProfileEvents_ReplicatedPartFailedFetches{pod=~\"$pod\",namespace=\"$namespace\"}[2m]))[$__rate_interval:15s] )","interval":"","legendFormat":"{{pod}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Replicated part failed fetches","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"cps","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"Number of times a data part was downloaded from replica of a ReplicatedMergeTree table.","fieldConfig":{"defaults":{"custom":{}},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":0,"y":15},"hiddenSeries":false,"id":57,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.3.5","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"max_over_time( (irate(ClickHouseProfileEvents_ReplicatedPartFetches{pod=~\"$pod\",namespace=\"$namespace\"}[2m]))[$__rate_interval:15s] )","interval":"","legendFormat":"{{pod}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Replicated part fetches","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"cps","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"Number of times we prefer to download already merged part from replica of ReplicatedMergeTree table instead of performing a merge ourself (usually we prefer doing a merge ourself to save network traffic). This happens when we have not all source parts to perform a merge or when the data part is old enough.","fieldConfig":{"defaults":{"custom":{}},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":12,"y":15},"hiddenSeries":false,"id":60,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.3.5","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"max_over_time( (irate(ClickHouseProfileEvents_ReplicatedPartFetchesOfMerged{pod=~\"$pod\",namespace=\"$namespace\"}[2m]))[$__rate_interval:15s] )","interval":"","legendFormat":"{{pod}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Replicated part fetches of merged","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"cps","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"Number of times data parts of ReplicatedMergeTree tables were successfully merged.","fieldConfig":{"defaults":{"custom":{}},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":0,"y":23},"hiddenSeries":false,"id":59,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.3.5","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"max_over_time( (irate(ClickHouseProfileEvents_ReplicatedPartMerges{pod=~\"$pod\",namespace=\"$namespace\"}[2m]))[$__rate_interval:15s] )","interval":"","legendFormat":"{{pod}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Replicated part merges","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"cps","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"Number of times a data part that we wanted doesn't exist on any replica (even on replicas that are offline right now). That data parts are definitely lost. This is normal due to asynchronous replication (if quorum inserts were not enabled), when the replica on which the data part was written was failed and when it became online after fail it doesn't contain that data part.","fieldConfig":{"defaults":{"custom":{}},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":12,"y":23},"hiddenSeries":false,"id":61,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.3.5","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"max_over_time( (irate(ClickHouseProfileEvents_ReplicatedDataLoss{pod=~\"$pod\",namespace=\"$namespace\"}[2m]))[$__rate_interval:15s] )","interval":"","legendFormat":"{{pod}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Replicated part data loss","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"cps","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}}],"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"refId":"A"}],"title":"Replicas","type":"row"},{"collapsed":true,"datasource":{"type":"prometheus","uid":"prometheus"},"gridPos":{"h":1,"w":24,"x":0,"y":43},"id":76,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"Number of launched background merges.","fieldConfig":{"defaults":{"custom":{}},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":0,"y":8},"hiddenSeries":false,"id":77,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.3.5","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"max_over_time( (irate(ClickHouseProfileEvents_Merge{pod=~\"$pod\",namespace=\"$namespace\"}[2m]))[$__rate_interval:15s] )","interval":"","legendFormat":"{{pod}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Merge","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"cps","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"Rows read for background merges. This is the number of rows before merge.","fieldConfig":{"defaults":{"custom":{}},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":12,"y":8},"hiddenSeries":false,"id":78,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.3.5","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"max_over_time( (irate(ClickHouseProfileEvents_MergedRows{pod=~\"$pod\",namespace=\"$namespace\"}[2m]))[$__rate_interval:15s] )","interval":"","legendFormat":"{{pod}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Merged rows","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"cps","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"Uncompressed bytes (for columns as they stored in memory) that was read for background merges. This is the number before merge.","fieldConfig":{"defaults":{"custom":{}},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":0,"y":16},"hiddenSeries":false,"id":79,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.3.5","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"max_over_time( (irate(ClickHouseProfileEvents_MergedUncompressedBytes{pod=~\"$pod\",namespace=\"$namespace\"}[2m]))[$__rate_interval:15s] )","interval":"","legendFormat":"{{pod}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Merge uncompressed bytes","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"binBps","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"Avg merge duration","fieldConfig":{"defaults":{"custom":{}},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":12,"y":16},"hiddenSeries":false,"id":80,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null as zero","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.3.5","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"increase(ClickHouseProfileEvents_MergesTimeMilliseconds{pod=~\"$pod\",namespace=\"$namespace\"}[$__rate_interval]) / increase(ClickHouseProfileEvents_Merge{pod=~\"$pod\",namespace=\"$namespace\"}[$__rate_interval])","interval":"","legendFormat":"{{pod}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Merge duration (avg)","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"µs","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false,"alignLevel":1}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"Number of rows INSERTed to MergeTree tables.","fieldConfig":{"defaults":{"custom":{}},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":0,"y":24},"hiddenSeries":false,"id":81,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.3.5","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"max_over_time( (irate(ClickHouseProfileEvents_MergeTreeDataWriterRows{pod=~\"$pod\",namespace=\"$namespace\"}[2m]))[$__rate_interval:15s] )","interval":"","legendFormat":"{{pod}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"MergeTree rows","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"cps","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"Number of blocks INSERTed to MergeTree tables. Each block forms a data part of level zero.","fieldConfig":{"defaults":{"custom":{}},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":12,"y":24},"hiddenSeries":false,"id":82,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.3.5","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"max_over_time( (irate(ClickHouseProfileEvents_MergeTreeDataWriterBlocks{pod=~\"$pod\",namespace=\"$namespace\"}[2m]))[$__rate_interval:15s] )","interval":"","legendFormat":"{{pod}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"MergeTree blocks","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"cps","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"Uncompressed bytes (for columns as they stored in memory) INSERTed to MergeTree tables.","fieldConfig":{"defaults":{"custom":{}},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":0,"y":32},"hiddenSeries":false,"id":83,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.3.5","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"max_over_time( (irate(ClickHouseProfileEvents_MergeTreeDataWriterUncompressedBytes{pod=~\"$pod\",namespace=\"$namespace\"}[2m]))[$__rate_interval:15s] )","interval":"","legendFormat":"{{pod}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"MergeTree uncompressed bytes","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"binBps","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"Bytes written to filesystem for data INSERTed to MergeTree tables.","fieldConfig":{"defaults":{"custom":{}},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":12,"y":32},"hiddenSeries":false,"id":84,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.3.5","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"max_over_time( (irate(ClickHouseProfileEvents_MergeTreeDataWriterCompressedBytes{pod=~\"$pod\",namespace=\"$namespace\"}[2m]))[$__rate_interval:15s] )","interval":"","legendFormat":"{{pod}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"MergeTree compressed bytes","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"binBps","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"Disk space reserved for currently running background merges. It is slightly more than the total size of currently merging parts.","fieldConfig":{"defaults":{"custom":{}},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":0,"y":40},"hiddenSeries":false,"id":123,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.3.5","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"ClickHouseMetrics_DiskSpaceReservedForMerge","interval":"","legendFormat":"{{pod}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Reserved space","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"bytes","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}}],"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"refId":"A"}],"title":"Merge","type":"row"},{"collapsed":true,"datasource":{"type":"prometheus","uid":"prometheus"},"gridPos":{"h":1,"w":24,"x":0,"y":44},"id":92,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"","fieldConfig":{"defaults":{"unitScale":true},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":0,"y":9},"hiddenSeries":false,"id":102,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"10.3.3","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"max_over_time( (irate(ClickHouseProfileEvents_UncompressedCacheHits{pod=~\"$pod\",namespace=\"$namespace\"}[2m]))[$__rate_interval:15s] )","interval":"","legendFormat":"{{pod}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Uncompressed Cache Hits","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"reqps","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"","fieldConfig":{"defaults":{"unitScale":true},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":12,"y":9},"hiddenSeries":false,"id":103,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"10.3.3","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"max_over_time( (irate(ClickHouseProfileEvents_UncompressedCacheMisses{pod=~\"$pod\",namespace=\"$namespace\"}[2m]))[$__rate_interval:15s] )","interval":"","legendFormat":"{{pod}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Uncompressed Cache Misses","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"reqps","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"","fieldConfig":{"defaults":{"unitScale":true},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":0,"y":17},"hiddenSeries":false,"id":104,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"10.3.3","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"max_over_time( (irate(ClickHouseProfileEvents_MarkCacheHits{pod=~\"$pod\",namespace=\"$namespace\"}[2m]))[$__rate_interval:15s] )","interval":"","legendFormat":"{{pod}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Mark Cache Hits","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"reqps","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"","fieldConfig":{"defaults":{"unitScale":true},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":12,"y":17},"hiddenSeries":false,"id":105,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"10.3.3","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"max_over_time( (irate(ClickHouseProfileEvents_MarkCacheMisses{pod=~\"$pod\",namespace=\"$namespace\"}[2m]))[$__rate_interval:15s] )","interval":"","legendFormat":"{{pod}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Mark Cache Misses","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"reqps","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}}],"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"refId":"A"}],"title":"Cache","type":"row"},{"collapsed":true,"datasource":{"type":"prometheus","uid":"prometheus"},"gridPos":{"h":1,"w":24,"x":0,"y":45},"id":90,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"The part is generating now, it is not in data_parts list.","fieldConfig":{"defaults":{"custom":{}},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":0,"y":10},"hiddenSeries":false,"id":114,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.3.5","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"ClickHouseMetrics_PartsTemporary{pod=~\"$pod\",namespace=\"$namespace\"}","interval":"","legendFormat":"{{pod}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Temporary","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"short","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"The part is in data_parts, but not used for SELECTs.","fieldConfig":{"defaults":{"custom":{}},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":12,"y":10},"hiddenSeries":false,"id":115,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.3.5","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"ClickHouseMetrics_PartsPreCommitted{pod=~\"$pod\",namespace=\"$namespace\"}","interval":"","legendFormat":"{{pod}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Pre commited","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"short","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"The part is in data_parts, but not used for SELECTs.","fieldConfig":{"defaults":{"custom":{}},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":0,"y":18},"hiddenSeries":false,"id":116,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.3.5","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"ClickHouseMetrics_PartsCommitted{pod=~\"$pod\",namespace=\"$namespace\"}","interval":"","legendFormat":"{{pod}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Commited","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"short","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"Not active data part, but could be used by only current SELECTs, could be deleted after SELECTs finishes.","fieldConfig":{"defaults":{"custom":{}},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":12,"y":18},"hiddenSeries":false,"id":117,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.3.5","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"ClickHouseMetrics_PartsOutdated{pod=~\"$pod\",namespace=\"$namespace\"}","interval":"","legendFormat":"{{pod}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Outdated","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"short","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"Not active data part with identity refcounter, it is deleting right now by a cleaner.","fieldConfig":{"defaults":{"custom":{}},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":0,"y":26},"hiddenSeries":false,"id":118,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.3.5","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"ClickHouseMetrics_PartsDeleting{pod=~\"$pod\",namespace=\"$namespace\"}","interval":"","legendFormat":"{{pod}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Deleting","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"short","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"Part was moved to another disk and should be deleted in own destructor.","fieldConfig":{"defaults":{"custom":{}},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":12,"y":26},"hiddenSeries":false,"id":119,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.3.5","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"ClickHouseMetrics_PartsDeleteOnDestroy{pod=~\"$pod\",namespace=\"$namespace\"}","interval":"","legendFormat":"{{pod}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Delete on destroy","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"short","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"Wide parts.","fieldConfig":{"defaults":{"custom":{}},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":0,"y":34},"hiddenSeries":false,"id":120,"legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.3.5","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"ClickHouseMetrics_PartsWide","interval":"","legendFormat":"{{pod}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Wide","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"short","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"Compact parts.","fieldConfig":{"defaults":{"custom":{}},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":12,"y":34},"hiddenSeries":false,"id":122,"legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.3.5","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"ClickHouseMetrics_PartsCompact","interval":"","legendFormat":"{{pod}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Compact","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"short","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"In-memory parts.","fieldConfig":{"defaults":{"custom":{}},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":0,"y":42},"hiddenSeries":false,"id":121,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.3.5","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"ClickHouseMetrics_PartsInMemory{pod=~\"$pod\",namespace=\"$namespace\"}","interval":"","legendFormat":"{{pod}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"In-memory","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"short","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}}],"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"refId":"A"}],"title":"Parts","type":"row"},{"collapsed":true,"datasource":{"type":"prometheus","uid":"prometheus"},"gridPos":{"h":1,"w":24,"x":0,"y":46},"id":94,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"Number of connections to remote servers sending data that was INSERTed into Distributed tables. Both synchronous and asynchronous mode.","fieldConfig":{"defaults":{"custom":{}},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":0,"y":11},"hiddenSeries":false,"id":112,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.3.5","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"ClickHouseMetrics_DistributedSend{pod=~\"$pod\",namespace=\"$namespace\"}","interval":"","legendFormat":"{{pod}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Send","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"short","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"Number of pending files to process for asynchronous insertion into Distributed tables. Number of files for every shard is summed.","fieldConfig":{"defaults":{"custom":{}},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":12,"y":11},"hiddenSeries":false,"id":113,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.3.5","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"ClickHouseMetrics_DistributedFilesToInsert{pod=~\"$pod\",namespace=\"$namespace\"}","interval":"","legendFormat":"{{pod}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Files to insert","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"short","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"Number of times the INSERT of a block to a Distributed table was throttled due to high number of pending bytes.","fieldConfig":{"defaults":{"custom":{}},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":0,"y":19},"hiddenSeries":false,"id":106,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.3.5","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"max_over_time( (irate(ClickHouseProfileEvents_DistributedDelayedInserts{pod=~\"$pod\",namespace=\"$namespace\"}[2m]))[$__rate_interval:15s] )","interval":"","legendFormat":"{{pod}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Distributed delayed inserts","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"reqps","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"Number of times the INSERT of a block to a Distributed table was rejected with 'Too many bytes' exception due to high number of pending bytes.","fieldConfig":{"defaults":{"custom":{}},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":12,"y":19},"hiddenSeries":false,"id":107,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.3.5","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"max_over_time( (irate(ClickHouseProfileEvents_DistributedRejectedInserts{pod=~\"$pod\",namespace=\"$namespace\"}[2m]))[$__rate_interval:15s] )","interval":"","legendFormat":"{{pod}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Distributed rejected inserts","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"reqps","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"Total number of milliseconds spent while the INSERT of a block to a Distributed table was throttled due to high number of pending bytes.","fieldConfig":{"defaults":{"custom":{}},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":0,"y":27},"hiddenSeries":false,"id":108,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null as zero","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.3.5","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"increase(ClickHouseProfileEvents_DistributedDelayedInsertsMilliseconds{pod=~\"$pod\",namespace=\"$namespace\"}[$__rate_interval]) / increase(ClickHouseProfileEvents_DistributedDelayedInserts{pod=~\"$pod\",namespace=\"$namespace\"}[$__rate_interval])","interval":"","legendFormat":"{{pod}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Distributed delayed inserts blocks (avg)","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"µs","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false,"alignLevel":1}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"Total count when distributed connection fails with retry","fieldConfig":{"defaults":{"custom":{}},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":12,"y":27},"hiddenSeries":false,"id":109,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.3.5","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"max_over_time( (irate(ClickHouseProfileEvents_DistributedConnectionFailTry{pod=~\"$pod\",namespace=\"$namespace\"}[2m]))[$__rate_interval:15s] )","interval":"","legendFormat":"{{pod}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Distributed connection fail try","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"reqps","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"Total count when distributed connection fails after all retries finished","fieldConfig":{"defaults":{"custom":{}},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":0,"y":35},"hiddenSeries":false,"id":110,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.3.5","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"max_over_time( (irate(ClickHouseProfileEvents_DistributedConnectionFailAtAll{pod=~\"$pod\",namespace=\"$namespace\"}[2m]))[$__rate_interval:15s] )","interval":"","legendFormat":"{{pod}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Distributed connection fail at all","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"reqps","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"","fieldConfig":{"defaults":{"custom":{}},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":12,"y":35},"hiddenSeries":false,"id":111,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.3.5","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"max_over_time( (irate(ClickHouseProfileEvents_DistributedSyncInsertionTimeoutExceeded{pod=~\"$pod\",namespace=\"$namespace\"}[2m]))[$__rate_interval:15s] )","interval":"","legendFormat":"{{pod}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Distributed sync insertation timeout","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"reqps","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}}],"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"refId":"A"}],"title":"Distributed","type":"row"},{"collapsed":true,"datasource":{"type":"prometheus","uid":"prometheus"},"gridPos":{"h":1,"w":24,"x":0,"y":47},"id":88,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"Number of active tasks in BackgroundProcessingPool (merges, mutations, or replication queue bookkeeping)","fieldConfig":{"defaults":{"custom":{}},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":0,"y":12},"hiddenSeries":false,"id":95,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.3.5","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"ClickHouseMetrics_BackgroundPoolTask{pod=~\"$pod\",namespace=\"$namespace\"}","interval":"","legendFormat":"{{pod}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"BackgroundPool task","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"short","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"Number of active tasks in BackgroundFetchesPool","fieldConfig":{"defaults":{"custom":{}},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":12,"y":12},"hiddenSeries":false,"id":96,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.3.5","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"ClickHouseMetrics_BackgroundFetchesPoolTask{pod=~\"$pod\",namespace=\"$namespace\"}","interval":"","legendFormat":"{{pod}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"BackgroundFetchesPool task","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"short","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"Number of active tasks in BackgroundProcessingPool for moves","fieldConfig":{"defaults":{"custom":{}},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":0,"y":20},"hiddenSeries":false,"id":97,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.3.5","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"ClickHouseMetrics_BackgroundMovePoolTask{pod=~\"$pod\",namespace=\"$namespace\"}","interval":"","legendFormat":"{{pod}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"BackgroundMovePool task","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"short","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"Number of active tasks in BackgroundSchedulePool. This pool is used for periodic ReplicatedMergeTree tasks, like cleaning old data parts, altering data parts, replica re-initialization, etc.","fieldConfig":{"defaults":{"custom":{}},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":12,"y":20},"hiddenSeries":false,"id":98,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.3.5","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"ClickHouseMetrics_BackgroundSchedulePoolTask{pod=~\"$pod\",namespace=\"$namespace\"}","interval":"","legendFormat":"{{pod}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"BackgroundSchedulePool task","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"short","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"Number of active tasks in BackgroundBufferFlushSchedulePool. This pool is used for periodic Buffer flushes","fieldConfig":{"defaults":{"custom":{}},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":0,"y":28},"hiddenSeries":false,"id":99,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.3.5","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"ClickHouseMetrics_BackgroundBufferFlushSchedulePoolTask{pod=~\"$pod\",namespace=\"$namespace\"}","interval":"","legendFormat":"{{pod}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"BackgroundBufferFlushSchedulePool task","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"short","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"Number of active tasks in BackgroundDistributedSchedulePool. This pool is used for distributed sends that is done in background.","fieldConfig":{"defaults":{"custom":{}},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":12,"y":28},"hiddenSeries":false,"id":100,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.3.5","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"ClickHouseMetrics_BackgroundDistributedSchedulePoolTask{pod=~\"$pod\",namespace=\"$namespace\"}","interval":"","legendFormat":"{{pod}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"BackgroundDistributedSchedulePool task","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"short","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"Number of active tasks in BackgroundProcessingPool for message streaming","fieldConfig":{"defaults":{"custom":{}},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":0,"y":36},"hiddenSeries":false,"id":101,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.3.5","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"ClickHouseMetrics_BackgroundMessageBrokerSchedulePoolTask{pod=~\"$pod\",namespace=\"$namespace\"}","interval":"","legendFormat":"{{pod}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"BackgroundMessageBrokerSchedulePool task","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"short","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}}],"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"refId":"A"}],"title":"Background pool","type":"row"},{"collapsed":true,"datasource":{"type":"prometheus","uid":"prometheus"},"gridPos":{"h":1,"w":24,"x":0,"y":48},"id":28,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"","fieldConfig":{"defaults":{"custom":{}},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":0,"y":13},"hiddenSeries":false,"id":30,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.3.5","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"max_over_time( (irate(ClickHouseProfileEvents_ZooKeeperInit{pod=~\"$pod\",namespace=\"$namespace\"}[2m]))[$__rate_interval:15s] )","interval":"","legendFormat":"{{pod}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Init","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"reqps","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"","fieldConfig":{"defaults":{"custom":{}},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":12,"y":13},"hiddenSeries":false,"id":31,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.3.5","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"max_over_time( (irate(ClickHouseProfileEvents_ZooKeeperTransactions{pod=~\"$pod\",namespace=\"$namespace\"}[2m]))[$__rate_interval:15s] )","interval":"","legendFormat":"{{pod}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Transactions","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"reqps","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"","fieldConfig":{"defaults":{"custom":{}},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":0,"y":21},"hiddenSeries":false,"id":32,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.3.5","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"max_over_time( (irate(ClickHouseProfileEvents_ZooKeeperList{pod=~\"$pod\",namespace=\"$namespace\"}[2m]))[$__rate_interval:15s] )","interval":"","legendFormat":"{{pod}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"List","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"reqps","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"","fieldConfig":{"defaults":{"custom":{}},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":12,"y":21},"hiddenSeries":false,"id":33,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.3.5","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"max_over_time( (irate(ClickHouseProfileEvents_ZooKeeperCreate{pod=~\"$pod\",namespace=\"$namespace\"}[2m]))[$__rate_interval:15s] )","interval":"","legendFormat":"{{pod}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Create","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"reqps","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"","fieldConfig":{"defaults":{"custom":{}},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":0,"y":29},"hiddenSeries":false,"id":34,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.3.5","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"max_over_time( (irate(ClickHouseProfileEvents_ZooKeeperRemove{pod=~\"$pod\",namespace=\"$namespace\"}[2m]))[$__rate_interval:15s] )","interval":"","legendFormat":"{{pod}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Remove","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"reqps","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"","fieldConfig":{"defaults":{"custom":{}},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":12,"y":29},"hiddenSeries":false,"id":35,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.3.5","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"max_over_time( (irate(ClickHouseProfileEvents_ZooKeeperExists{pod=~\"$pod\",namespace=\"$namespace\"}[2m]))[$__rate_interval:15s] )","interval":"","legendFormat":"{{pod}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Exists","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"reqps","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"","fieldConfig":{"defaults":{"custom":{}},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":0,"y":37},"hiddenSeries":false,"id":36,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.3.5","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"max_over_time( (irate(ClickHouseProfileEvents_ZooKeeperGet{pod=~\"$pod\",namespace=\"$namespace\"}[2m]))[$__rate_interval:15s] )","interval":"","legendFormat":"{{pod}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Get","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"reqps","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"","fieldConfig":{"defaults":{"custom":{}},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":12,"y":37},"hiddenSeries":false,"id":37,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.3.5","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"max_over_time( (irate(ClickHouseProfileEvents_ZooKeeperSet{pod=~\"$pod\",namespace=\"$namespace\"}[2m]))[$__rate_interval:15s] )","interval":"","legendFormat":"{{pod}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Set","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"reqps","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"","fieldConfig":{"defaults":{"custom":{}},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":0,"y":45},"hiddenSeries":false,"id":38,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.3.5","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"max_over_time( (irate(ClickHouseProfileEvents_ZooKeeperMulti{pod=~\"$pod\",namespace=\"$namespace\"}[2m]))[$__rate_interval:15s] )","interval":"","legendFormat":"{{pod}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Multi","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"reqps","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"","fieldConfig":{"defaults":{"custom":{}},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":12,"y":45},"hiddenSeries":false,"id":39,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.3.5","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"max_over_time( (irate(ClickHouseProfileEvents_ZooKeeperCheck{pod=~\"$pod\",namespace=\"$namespace\"}[2m]))[$__rate_interval:15s] )","interval":"","legendFormat":"{{pod}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Check","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"reqps","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"","fieldConfig":{"defaults":{"custom":{}},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":0,"y":53},"hiddenSeries":false,"id":40,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.3.5","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"max_over_time( (irate(ClickHouseProfileEvents_ZooKeeperClose{pod=~\"$pod\",namespace=\"$namespace\"}[2m]))[$__rate_interval:15s] )","interval":"","legendFormat":"{{pod}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Close","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"reqps","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"","fieldConfig":{"defaults":{"custom":{}},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":12,"y":53},"hiddenSeries":false,"id":41,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.3.5","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"max_over_time( (irate(ClickHouseProfileEvents_ZooKeeperWatchResponse{pod=~\"$pod\",namespace=\"$namespace\"}[2m]))[$__rate_interval:15s] )","interval":"","legendFormat":"{{pod}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Watch response","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"reqps","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"","fieldConfig":{"defaults":{"custom":{}},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":0,"y":61},"hiddenSeries":false,"id":42,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.3.5","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"max_over_time( (irate(ClickHouseProfileEvents_ZooKeeperUserExceptions{pod=~\"$pod\",namespace=\"$namespace\"}[2m]))[$__rate_interval:15s] )","interval":"","legendFormat":"{{pod}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"User Exceptions","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"cps","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"","fieldConfig":{"defaults":{"custom":{}},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":12,"y":61},"hiddenSeries":false,"id":43,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.3.5","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"max_over_time( (irate(ClickHouseProfileEvents_ZooKeeperHardwareExceptions{pod=~\"$pod\",namespace=\"$namespace\"}[2m]))[$__rate_interval:15s] )","interval":"","legendFormat":"{{pod}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Hardware Exceptions","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"cps","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"","fieldConfig":{"defaults":{"custom":{}},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":0,"y":69},"hiddenSeries":false,"id":46,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.3.5","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"max_over_time( (irate(ClickHouseProfileEvents_ZooKeeperBytesReceived{pod=~\"$pod\",namespace=\"$namespace\"}[2m]))[$__rate_interval:15s] )","interval":"","legendFormat":"{{pod}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Bytes received","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"binBps","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"","fieldConfig":{"defaults":{"custom":{}},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":12,"y":69},"hiddenSeries":false,"id":45,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.3.5","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"max_over_time( (irate(ClickHouseProfileEvents_ZooKeeperBytesSent{pod=~\"$pod\",namespace=\"$namespace\"}[2m]))[$__rate_interval:15s] )","interval":"","legendFormat":"{{pod}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Bytes sent","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"binBps","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"Number of sessions (connections) to ZooKeeper. Should be no more than one, because using more than one connection to ZooKeeper may lead to bugs due to lack of linearizability (stale reads) that ZooKeeper consistency model allows.","fieldConfig":{"defaults":{"custom":{}},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":0,"y":77},"hiddenSeries":false,"id":48,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.3.5","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"ClickHouseMetrics_ZooKeeperSession{pod=~\"$pod\",namespace=\"$namespace\"}","interval":"","legendFormat":"{{pod}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Sessions","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"short","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"Number of ephemeral nodes hold in ZooKeeper.","fieldConfig":{"defaults":{"custom":{}},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":12,"y":77},"hiddenSeries":false,"id":47,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.3.5","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"ClickHouseMetrics_EphemeralNode{pod=~\"$pod\",namespace=\"$namespace\"}","interval":"","legendFormat":"{{pod}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Ephemeral Node","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"short","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"Number of watches (event subscriptions) in ZooKeeper.","fieldConfig":{"defaults":{"custom":{}},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":0,"y":85},"hiddenSeries":false,"id":50,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.3.5","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"ClickHouseMetrics_ZooKeeperWatch{pod=~\"$pod\",namespace=\"$namespace\"}","interval":"","legendFormat":"{{pod}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Watches","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"short","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"Number of requests to ZooKeeper in fly.","fieldConfig":{"defaults":{"custom":{}},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":12,"y":85},"hiddenSeries":false,"id":49,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.3.5","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"ClickHouseMetrics_ZooKeeperRequest{pod=~\"$pod\",namespace=\"$namespace\"}","interval":"","legendFormat":"{{pod}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Requests","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"short","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"prometheus"},"description":"Number of times an error happened while trying to remove ephemeral node. This is not an issue, because our implementation of ZooKeeper library guarantee that the session will expire and the node will be removed.","fieldConfig":{"defaults":{"custom":{}},"overrides":[]},"fill":0,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":0,"y":93},"hiddenSeries":false,"id":70,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":2,"maxDataPoints":200,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.3.5","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"exemplar":false,"expr":"max_over_time( (irate(ClickHouseProfileEvents_CannotRemoveEphemeralNode{pod=~\"$pod\",namespace=\"$namespace\"}[2m]))[$__rate_interval:15s] )","interval":"","legendFormat":"{{pod}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Remove ephemeral node failed","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transparent":true,"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"$$hashKey":"object:176","format":"cps","label":"","logBase":1,"show":true},{"$$hashKey":"object:177","format":"short","logBase":1,"show":true}],"yaxis":{"align":false}}],"targets":[{"datasource":{"type":"prometheus","uid":"prometheus"},"refId":"A"}],"title":"Zookeeper","type":"row"}],"refresh":"30s","schemaVersion":39,"tags":["ClickHouse","DB"],"templating":{"list":[{"current":{"selected":false,"text":"Prometheus","value":"prometheus"},"hide":0,"includeAll":false,"label":"DataSource","multi":false,"name":"datasource","options":[],"query":"prometheus","queryValue":"","refresh":1,"regex":"","skipUrlSync":false,"type":"datasource"},{"current":{"selected":false,"text":"kdp-data","value":"kdp-data"},"datasource":{"type":"prometheus","uid":"${datasource}"},"definition":"label_values(ClickHouseAsyncMetrics_Uptime,namespace)","hide":0,"includeAll":false,"multi":false,"name":"namespace","options":[],"query":{"qryType":1,"query":"label_values(ClickHouseAsyncMetrics_Uptime,namespace)","refId":"PrometheusVariableQueryEditor-VariableQuery"},"refresh":1,"regex":"","skipUrlSync":false,"sort":1,"type":"query"},{"current":{"selected":true,"text":["All"],"value":["$__all"]},"datasource":{"type":"prometheus","uid":"prometheus"},"definition":"label_values(ClickHouseAsyncMetrics_Uptime,pod)","hide":0,"includeAll":true,"label":"Pod","multi":true,"name":"pod","options":[],"query":{"qryType":1,"query":"label_values(ClickHouseAsyncMetrics_Uptime,pod)","refId":"PrometheusVariableQueryEditor-VariableQuery"},"refresh":1,"regex":"","skipUrlSync":false,"sort":5,"tagValuesQuery":"","tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-15m","to":"now"},"timepicker":{},"timezone":"","title":"ClickHouse","uid":"clickhouse","version":4,"weekStart":""}"#
								}
							}
						},
						{
							type: "bdos-ingress"
							properties: {
								rules: [
									{
										host: context["name"] + "-" + context["namespace"] + "." + context["ingress.root_domain"]
										paths: [
											{
												path:        "/"
												servicePort: 8123
												serviceName: context.name
											},
										]
									},
								]
								tls: [
									{
										hosts: [
											context["name"] + "-" + context["namespace"] + "." + context["ingress.root_domain"],
										]
										tlsSecretName: context["ingress.tls_secret_name"]
									},
								]
							}
						},
					]
				},
				{
					"name": "\(context.name)-config"
					"type": "k8s-objects"
					"properties": {
						"objects": [
							{
								"apiVersion": "bdc.kdp.io/v1alpha1"
								"kind":       "ContextSetting"
								"metadata": {
									"namespace": context.namespace
									"name":      "\(context.bdc)-\(context.name)"
									"annotations": {
										"setting.ctx.bdc.kdp.io/type":   "clickhouse"
										"setting.ctx.bdc.kdp.io/origin": "system"
									}
								}
								"spec": {
									"name": "clickhouse-context"
									"type": "clickhouse"
									"properties": {
										"host":     "clickhouse.\(context.namespace).svc.cluster.local:8123"
										"hostname": "clickhouse.\(context.namespace).svc.cluster.local"
										"port":     "8123"
									}
								}
							}
						]
					}
				},
			]
			policies: [
				{
					name: "shared-resource"
					type: "shared-resource"
					properties: {
						rules: [
							{
								selector: {
									traitTypes: [
										"bdos-grafana-dashboard",
									]
								}
							},
						]
					}
				}
			]
		}
	}
	parameter: {
		// +ui:title=组件依赖
		// +ui:order=1
		dependencies: {
			// +ui:description=Zookeeper 地址
			// +ui:order=1
			// +err:options={"required":"请先安装 Zookeeper"}
			zookeeperHost: string
		}
    // +ui:description=分片数
		// +ui:order=2
		// +minimum=1
    shards: *1 | int
		// +ui:description=副本数
		// +ui:order=3
		// +minimum=1
		replicaCount: *1 | int
    // +ui:description=存储大小
		// +ui:order=4
		// +pattern=^[1-9]\d*(Mi|Gi)$
		// +err:options={"pattern":"请输入正确的存储格式，如1024Mi, 1Gi"}
    storageSize: *"20Gi" | string
		// +ui:description=资源规格
		// +ui:order=5
		resources: {
			// +ui:description=预留
			// +ui:order=1
			requests: {
				// +ui:description=CPU
				// +ui:order=1
				// +pattern=^(\d+\.\d{1,3}?|[1-9]\d*m?)$
				// +err:options={"pattern":"请输入正确的cpu格式，如1, 1000m"}
				cpu: *"0.1" | string
				// +ui:description=内存
				// +ui:order=2
				// +pattern=^[1-9]\d*(Mi|Gi)$
				// +err:options={"pattern":"请输入正确的内存格式，如1024Mi, 1Gi"}
				memory: *"512Mi" | string
			}
			// +ui:description=限制
			// +ui:order=2
			limits: {
				// +ui:description=CPU
				// +ui:order=1
				// +pattern=^(\d+\.\d{1,3}?|[1-9]\d*m?)$
				// +err:options={"pattern":"请输入正确的cpu格式，如1, 1000m"}
				cpu: *"4.0" | string
				// +ui:description=内存
				// +ui:order=2
				// +pattern=^[1-9]\d*(Mi|Gi)$
				// +err:options={"pattern":"请输入正确的内存格式，如1024Mi, 1Gi"}
				memory: *"4096Mi" | string
			}
		}
		// +ui:description=clickhouse管理员设置
		// +ui:order=6
		auth: {
			// +ui:description=管理员用户名
			// +ui:order=1
			username: *"default" | string
			// +ui:description=管理员密码。部署后无法在此页面更改密码。
			// +ui:options={"showPassword": true}
			// +ui:order=2
			password: *"ckdba.123" | string
		}
		// +ui:description=Helm Chart 版本号
		// +ui:order=100
		// +ui:options={"disabled":true}
		chartVersion: *"6.0.2" | string
		// +ui:description=镜像版本
		// +ui:order=101
		// +ui:options={"disabled":true}
		imageTag: *"24.3.2-debian-12-r2" | string
	}
}
